CarND: Advanced Lane Finding

Project: Build a advanced lane boundaries in a video from a front-facing camera on a car

In this notebook, a template is provided for you to implement your functionality in stages, which is required to successfully complete this project. If additional code is required that cannot be included in the notebook, be sure that the Python code is successfully imported and included in your submission if necessary.

The camera calibration images, test road images, and project videos are available in the project repository.

The goals / steps of this project are the following:

  • Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
  • Apply a distortion correction to raw images.
  • Use color transforms, gradients, etc., to create a thresholded binary image.
  • Apply a perspective transform to rectify binary image ("birds-eye view").
  • Detect lane pixels and fit to find the lane boundary.
  • Determine the curvature of the lane and vehicle position with respect to center.
  • Warp the detected lane boundaries back onto the original image.
  • Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

Step 0: Camera Calibration


1. Prepare work for camera calibration: etc. files load, packages import, and so on

In [1]:
#importing some useful packages
import os
import pickle
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import cv2
import numpy as np
import random 
%matplotlib inline

print('Loading data...')
print()

file_dir = 'camera_cal/'
cal_filelist = os.listdir(file_dir)

print("calibration files: ", cal_filelist)

index = random.randint(0, len(cal_filelist)-1)
tmp_img = cv2.imread(file_dir + cal_filelist[index], cv2.COLOR_BGR2RGB)
tmp_gray = cv2.cvtColor(tmp_img, cv2.COLOR_BGR2GRAY)
plt.imshow(tmp_img)

print("calibration image shape:   {}".format(tmp_img.shape) )
print("length of calibration image set: {} samples".format(len(cal_filelist)))

print()
print('Done ...')
Loading data...

calibration files:  ['calibration1.jpg', 'calibration10.jpg', 'calibration11.jpg', 'calibration12.jpg', 'calibration13.jpg', 'calibration14.jpg', 'calibration15.jpg', 'calibration16.jpg', 'calibration17.jpg', 'calibration18.jpg', 'calibration19.jpg', 'calibration2.jpg', 'calibration20.jpg', 'calibration3.jpg', 'calibration4.jpg', 'calibration5.jpg', 'calibration6.jpg', 'calibration7.jpg', 'calibration8.jpg', 'calibration9.jpg']
calibration image shape:   (721, 1281, 3)
length of calibration image set: 20 samples

Done ...

2. Finding Corners and draw

In [2]:
# corners inside the image
nx = 9
ny = 6

#Camera calibration
objpoints = []
imgpoints = []

objp = np.zeros((nx*ny, 3),np.float32)
objp[:, :2] = np.mgrid[0:nx, 0:ny].T.reshape(-1, 2)

import glob
# read in and make a list of images, this returns images' path list
images_list = glob.glob('camera_cal/calibration*.jpg')
#print(len(images))
#print((images))

# Convert to grayscale
for fname in images_list:
    tmp_img = cv2.imread(fname)
    tmp_gray = cv2.cvtColor(tmp_img, cv2.COLOR_BGR2GRAY)
    
    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(tmp_gray, (nx, ny), None)
    if ret == True:
        imgpoints.append(corners)
        objpoints.append(objp)
        img = cv2.drawChessboardCorners(tmp_img, (nx, ny), corners, ret)  
        plt.imshow(img)
        plt.show()
    else:
        print('findChessboardCorners error, return :',ret)
findChessboardCorners error, return : False
findChessboardCorners error, return : False
findChessboardCorners error, return : False
In [3]:
print(tmp_img.shape[0:2])
print(tmp_gray.shape[::-1])
img_size = (tmp_img.shape[1], tmp_img.shape[0])
print(img_size)
(720, 1280)
(1280, 720)
(1280, 720)

3. Test and Apply a distortion correction to raw images

In [4]:
# Test undistortion on an image
test_img = cv2.imread('camera_cal/calibration2.jpg')

# performs the camera calibration, image distortion correction and 
# returns the undistorted image
ret, mtx, dist, rvecs, tvecs, = cv2.calibrateCamera(objpoints, imgpoints, test_img.shape[0:2], None, None)
dst = cv2.undistort(test_img, mtx, dist, None, mtx)

# Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
dist_pickle = {}
dist_pickle["mtx"] = mtx
dist_pickle["dist"] = dist
pickle.dump( dist_pickle, open( "output_images/dist_pickle.p", "wb" ) )

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 12))
f.tight_layout()
ax1.imshow(test_img)
ax1.set_title('Original Image', fontsize=10)
ax2.imshow(dst)
ax2.set_title('Undistorted Image', fontsize=10)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
    

Perspective Transform

In [5]:
# Perspective Transform
def warp(image, src, dst):
    img_size = (image.shape[1], image.shape[0] )

    M = cv2.getPerspectiveTransform(src, dst)
    #Minv = cv2.getPerspectiveTransform(dst, src)
    #print('transform matrix is :',M)
    warped = cv2.warpPerspective(image, M, img_size, flags=cv2.INTER_LINEAR)
    return warped

src = np.float32(
    [[264, 101],
     [270, 142],
     [167, 112],
     [168, 69]])

dst = np.float32(
    [[264, 101],
     [264, 142],
     [167, 142],
     [167, 101]])
    
stopsign = cv2.imread('output_images/stop.png')
stopsign_img = cv2.cvtColor(stopsign, cv2.COLOR_BGR2RGB)
plt.imshow(stopsign_img)

plt.plot(264, 101, '.') #top right
plt.plot(270, 142, '.') #bottom right
plt.plot(167, 112, '.') #bottom left
plt.plot(168, 69, '.') #top left

trans_img = warp(stopsign_img, src, dst)

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(4, 4))
f.tight_layout()
ax1.imshow(stopsign_img)
ax1.set_title('Original Image', fontsize=10)
ax2.imshow(trans_img)
ax2.set_title('Perspective Transform Image', fontsize=10)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

read camera matrix and distortion coefficients

In [6]:
# Read in the saved camera matrix and distortion coefficients
dist_pickle = pickle.load( open( "output_images/dist_pickle.p", "rb" ) )
mtx = dist_pickle["mtx"]
dist = dist_pickle["dist"]

undistort, find chess corners, draw chess corners, perspective transform, warpperspective combined.

In [7]:
# undistort, find chess corners, draw chess corners, perspective transform, warpperspective combined.
def corners_unwarp(img, nx, ny, mtx, dist):
    # 1) Undistort using mtx and dist
    undis_img = cv2.undistort(img, mtx, dist, None, mtx)
    # 2) Convert to grayscale
    gray = cv2.cvtColor(undis_img, cv2.COLOR_BGR2GRAY)
    # 3) Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)
    
    # 4) If corners found: 
    if ret == True:
        img = cv2.drawChessboardCorners(undis_img, (nx, ny), corners, ret)
        src_point = np.float32([corners[0], corners[nx-1], corners[-1], corners[-nx] ])

        offset = 100
        img_size = (img.shape[1], img.shape[0])
        dst_point = np.float32([[offset, offset], 
                                [img_size[0]-offset, offset], 
                                [img_size[0]-offset, img_size[1]-offset], 
                                [offset, img_size[1]-offset]])
        M = cv2.getPerspectiveTransform(src_point, dst_point)
        warpedimg = cv2.warpPerspective(undis_img, M, img_size, flags=cv2.INTER_LINEAR)
        return warpedimg

warp1 = cv2.imread('camera_cal/calibration7.jpg')
top_down_img = corners_unwarp(warp1, nx, ny, mtx, dist)

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 12))
f.tight_layout()
ax1.imshow(warp1)
ax1.set_title('Original Image', fontsize=10)
ax2.imshow(top_down_img)
ax2.set_title('Perspective Transform Image', fontsize=10)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Use color transforms, gradients, etc., to create a thresholded binary image.

In [8]:
# sobel gradient function
def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0, 255)):  #(20,100)
    #  Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    
    if orient == 'x':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel))
    if orient == 'y':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel))
    
    # Scale to 8-bit (0 - 255) then convert to type = np.uint8
    scaled_sobel = np.uint8(255 * (abs_sobel/np.max(abs_sobel)))
    
    #Create a mask of 1's where the scaled gradient magnitude is > thresh_min and < thresh_max
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[( scaled_sobel >= thresh[0]) & ( scaled_sobel <= thresh[1])] = 1
    return binary_output

def mag_thresh(image, sobel_kernel=3, mag_thresh=(0, 255)):
    # Calculate gradient magnitude
    # Apply threshold
    return mag_binary

def dir_threshold(image, sobel_kernel=3, thresh=(0, np.pi/2)):
    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    abs_sobelx = np.absolute(sobelx)
    abs_sobely = np.absolute(sobely)
    absgraddir = np.arctan2(abs_sobely, abs_sobelx)
    binary_output = np.zeros_like(absgraddir)
    binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1
    return binary_output

● explore HLS color space

In [9]:
# HLS color space function
def hls_select(img, thresh=(90, 255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    s_channel = hls[:,:,2]
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= thresh[0]) & (s_channel <= thresh[1])] = 1
    return s_binary

timg = cv2.imread('test_images/straight_lines1.jpg') #straight_lines1.jpg
t_img = cv2.cvtColor(timg, cv2.COLOR_BGR2RGB)

s_binary = hls_select(t_img, thresh=(170, 255))

sxbinary = abs_sobel_thresh(t_img, orient='x', sobel_kernel=5, thresh=(20, 100))
color_binary = np.dstack(( np.zeros_like(sxbinary), sxbinary, s_binary))*255

print('s_binary shape:',s_binary.shape )
print('sxbinary shape:',sxbinary.shape )

combined_binary = np.zeros_like(sxbinary)
combined_binary[(s_binary == 1) | (sxbinary == 1)] = 1

# Plotting thresholded images
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))
ax1.set_title('Stacked thresholds')
ax1.imshow(color_binary)

ax2.set_title('Combined S channel and gradient thresholds')
ax2.imshow(combined_binary, cmap='gray')
s_binary shape: (720, 1280)
sxbinary shape: (720, 1280)
Out[9]:
<matplotlib.image.AxesImage at 0x1cd07d9df28>

● explore undistorted Image with src point drawn

In [10]:
def region_of_interest(img, vertices):
    mask = np.zeros_like(img)
    if(len(img.shape) > 2):
        channel_count = img.shape[2]
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255
    
    cv2.fillPoly(mask, vertices, ignore_mask_color)
    
    masked_image = cv2.bitwise_and(img, mask)
    return masked_image


img_size_rev = (t_img.shape[0], t_img.shape[1])
img_size = (t_img.shape[1], t_img.shape[0])
print(t_img.shape)
vertices = np.float32([
    [204, img_size[1]], #
    [585, 460],
    [705, 460],
    [1110, img_size[1]] #1110
])

undis_img = cv2.undistort(t_img, mtx, dist, None, mtx)

# Plotting thresholded images
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))
ax1.set_title('original image')
ax1.imshow(t_img)
xx = [vertices[0][0], vertices[1][0], vertices[2][0], vertices[3][0], vertices[0][0]]
yy = [vertices[0][1], vertices[1][1], vertices[2][1], vertices[3][1], vertices[0][1]]
plt.plot(xx, yy, 'r--', lw=2.0)

ax2.set_title('Undistorted Image with src point drawn ') #with src point drawn
ax2.imshow(undis_img)
(720, 1280, 3)
Out[10]:
<matplotlib.image.AxesImage at 0x1cd04878da0>
In [11]:
#im = region_of_interest(t_img, vertices)
#plt.imshow(im)

● explore perspective transform and wraped image

In [12]:
# undistort, perspective transform, warpperspective combined.
def images_warp(img, src_point, dst_point):
    img_size = (img.shape[1], img.shape[0])
    M = cv2.getPerspectiveTransform(src_point, dst_point)
    warpedimg = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR) #cv2.INTER_NEAREST  cv2.INTER_LINEAR
    return warpedimg

#offset = 100
src_pts = vertices     # [205, img_size[1]],   [595, 450],   [685, 450],   [1105, img_size[1]]
dst_pts = np.float32([
    [320, img_size[1]],
    [320, 0], 
    [960, 0], 
    [960, img_size[1]] ])

warp_img = images_warp(undis_img, src_pts, dst_pts)
# Plotting thresholded images
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))
ax1.set_title('Undistorted Image with src point drawn')
ax1.imshow(undis_img)
xx = [src_pts[0][0], src_pts[1][0], src_pts[2][0], src_pts[3][0], src_pts[0][0]]
yy = [src_pts[0][1], src_pts[1][1], src_pts[2][1], src_pts[3][1], src_pts[0][1]]
ax1.plot(xx, yy, 'r--', lw=2.0)

ax2.set_title('Warped result with dest. point drawn')
ax2.imshow(warp_img)
xx = [dst_pts[0][0], dst_pts[1][0], dst_pts[2][0], dst_pts[3][0], dst_pts[0][0]]
yy = [dst_pts[0][1], dst_pts[1][1], dst_pts[2][1], dst_pts[3][1], dst_pts[0][1]]
ax2.plot(xx, yy, 'r--', lw=2.0)
Out[12]:
[<matplotlib.lines.Line2D at 0x1cd048eec88>]

Apply and explore a perspective transform to rectify binary image ("birds-eye view")

In [13]:
gimg = cv2.imread('test_images/straight_lines1.jpg')
g_img = cv2.cvtColor(gimg, cv2.COLOR_BGR2RGB)

undis_gimg = cv2.undistort(g_img, mtx, dist, None, mtx)

s_binary_g = hls_select(undis_gimg, thresh=(170, 255))
sxbinary_g = abs_sobel_thresh(undis_gimg, orient='x', sobel_kernel=5, thresh=(20, 100))

combined_binary_g = np.zeros_like(sxbinary_g)
combined_binary_g[(s_binary_g == 1) | (sxbinary_g == 1)] = 1

warp_img_g = images_warp(combined_binary_g, src_pts, dst_pts)

# Plotting thresholded images
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))
ax1.set_title('Undistorted Image with src point drawn')
ax1.imshow(combined_binary_g, cmap='gray')
xx = [src_pts[0][0], src_pts[1][0], src_pts[2][0], src_pts[3][0], src_pts[0][0]]
yy = [src_pts[0][1], src_pts[1][1], src_pts[2][1], src_pts[3][1], src_pts[0][1]]
ax1.plot(xx, yy, 'r--', lw=2.0)

ax2.set_title('Warped binary image with dest. point drawn ')
ax2.imshow(warp_img_g, cmap='gray')
xx = [dst_pts[0][0], dst_pts[1][0], dst_pts[2][0], dst_pts[3][0], dst_pts[0][0]]
yy = [dst_pts[0][1], dst_pts[1][1], dst_pts[2][1], dst_pts[3][1], dst_pts[0][1]]
ax2.plot(xx, yy, 'r--', lw=2.0)
Out[13]:
[<matplotlib.lines.Line2D at 0x1cd046470f0>]

explore curved lines

In [14]:
himg = cv2.imread('test_images/test3.jpg')
h_img = cv2.cvtColor(himg, cv2.COLOR_BGR2RGB)

undis_himg = cv2.undistort(h_img, mtx, dist, None, mtx)

s_binary_h = hls_select(undis_himg, thresh=(170, 255))
sxbinary_h = abs_sobel_thresh(undis_himg, orient='x', sobel_kernel=5, thresh=(20, 100))

combined_binary_h = np.zeros_like(sxbinary_h)
combined_binary_h[(s_binary_h == 1) | (sxbinary_h == 1)] = 1

warp_img_h = images_warp(combined_binary_h, src_pts, dst_pts)

# Plotting thresholded images
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))
ax1.set_title('Undistorted Image with src point drawn')
ax1.imshow(undis_himg, cmap='gray')
xx = [src_pts[0][0], src_pts[1][0], src_pts[2][0], src_pts[3][0], src_pts[0][0]]
yy = [src_pts[0][1], src_pts[1][1], src_pts[2][1], src_pts[3][1], src_pts[0][1]]
ax1.plot(xx, yy, 'r--', lw=2.0)

ax2.set_title('Warped binary image with dest. point drawn ')
ax2.imshow(warp_img_h, cmap='gray')
xx = [dst_pts[0][0], dst_pts[1][0], dst_pts[2][0], dst_pts[3][0], dst_pts[0][0]]
yy = [dst_pts[0][1], dst_pts[1][1], dst_pts[2][1], dst_pts[3][1], dst_pts[0][1]]
ax2.plot(xx, yy, 'r--', lw=2.0)
Out[14]:
[<matplotlib.lines.Line2D at 0x1cd06a1b4a8>]

Detect lane pixels and fit to find the lane boundary.

get histogram

In [15]:
def getHistogram(warp_img):
    # Take a histogram of the bottom half of the image
    #translated the histogram by `dst_pts[0][0]-100 = 220` pix from left to right,
    left_x_ind = int(dst_pts[0][0]-200)  #120
    right_x_ind = int(dst_pts[3][0]+200) #1160
    histogram = np.sum(warp_img[int(warp_img.shape[0]/2):, left_x_ind:right_x_ind], axis=0)
    #histogram = np.sum(warp_img[int(warp_img.shape[0]/2):, :], axis=0)
    return histogram

histogram = getHistogram(warp_img_h)
print(histogram.shape)
plt.plot(histogram)
plt.show()
(1040,)

fit the 2nd order polynomial

In [16]:
def fit_poly(binary_warped, histogram):
    # Create an output image to draw on and  visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]/2) 
    leftx_base = np.argmax(histogram[:midpoint]) + 120
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint + 120

    # Choose the number of sliding windows
    nwindows = 9
    # Set height of windows
    window_height = np.int(binary_warped.shape[0]/nwindows)

    # Identify the x and y positions of all nonzero pixels in the image
    nonezero = binary_warped.nonzero()
    nonezeroy = np.array(nonezero[0])
    nonezerox = np.array(nonezero[1])
    # Current positions to be updated for each window
    leftx_current = leftx_base 
    rightx_current = rightx_base 

    # Set the width of the windows +/- margin
    margin = 95
    # Set minimum number of pixels found to recenter window
    minpix = 50
    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_high = binary_warped.shape[0] - window * window_height 
        win_y_low =  binary_warped.shape[0] - (window+1)*window_height 
        win_xleft_low = leftx_current - margin 
        win_xleft_high = leftx_current + margin 
        win_xright_low = rightx_current - margin  
        win_xright_high = rightx_current + margin 
        # Draw the windows on the visualization image
        cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),(0,255,0), 3)
        cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),(0,255,0), 3) 
        #cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),(0,0,255), 3) 
        # Identify the nonzero pixels in x and y within the window
        good_left_inds = ((nonezeroy>=win_y_low)&(nonezeroy<win_y_high) & (nonezerox>=win_xleft_low)&(nonezerox<win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonezeroy>=win_y_low)&(nonezeroy<=win_y_high) & (nonezerox>=win_xright_low)&(nonezerox<=win_xright_high)).nonzero()[0]
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        # If you found > minpix pixels, recenter next window on their mean position
        if(len(good_left_inds) > minpix):
            leftx_current = np.int(np.mean(nonezerox[good_left_inds]))
        if(len(good_right_inds) > minpix):
            rightx_current = np.int(np.mean(nonezerox[good_right_inds]))
            
    # Concatenate the arrays of indices
    left_lane_inds = np.concatenate(left_lane_inds)
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    #leftx = nonezerox[left_lane_inds]
    #lefty = nonezeroy[left_lane_inds]
    #rightx = nonezerox[right_lane_inds]
    #righty = nonezeroy[right_lane_inds]

    # Fit a second order polynomial to each
    #left_fit = np.polyfit(lefty, leftx, 2)
    #right_fit = np.polyfit(righty, rightx, 2)
    return out_img, left_lane_inds, right_lane_inds    #out_img, left_lane_inds, right_lane_inds, 

Visualization the lines

In [17]:
# Generate x and y values for plotting

out_img, left_lane_inds, right_lane_inds = fit_poly(warp_img_h, histogram)

nonezero = warp_img_h.nonzero()
nonezeroy = np.array(nonezero[0])
nonezerox = np.array(nonezero[1])

# Extract left and right line pixel positions
leftx = nonezerox[left_lane_inds]
lefty = nonezeroy[left_lane_inds]
rightx = nonezerox[right_lane_inds]
righty = nonezeroy[right_lane_inds]

# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
    
ploty = np.linspace(0, warp_img_h.shape[0], warp_img_h.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]

out_img[nonezeroy[left_lane_inds], nonezerox[left_lane_inds]] = [255, 0, 0]
out_img[nonezeroy[right_lane_inds], nonezerox[right_lane_inds]] = [0, 0, 255]
plt.imshow(out_img)
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
plt.xlim(0, 1280)
plt.ylim(720, 0)
Out[17]:
(720, 0)
In [18]:
# Create an image to draw on and an image to show the selection window
out_img = np.dstack((warp_img_h, warp_img_h, warp_img_h))*255
window_img = np.zeros_like(out_img)
# Color in left and right line pixels
out_img[nonezeroy[left_lane_inds], nonezerox[left_lane_inds]] = [255, 0, 0]
out_img[nonezeroy[right_lane_inds], nonezerox[right_lane_inds]] = [0, 0, 255]
margin = 80
# Generate a polygon to illustrate the search window area
# And recast the x and y points into usable format for cv2.fillPoly()
left_line_window1 = np.array([ np.transpose(np.vstack([left_fitx-margin, ploty]))])
left_line_window2 = np.array([ np.flipud(np.transpose(np.vstack([left_fitx+margin, ploty])))])
left_line_pts = np.hstack((left_line_window1, left_line_window2))
right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin, ploty])))])
right_line_pts = np.hstack((right_line_window1, right_line_window2))

mid_line_pts = np.hstack((left_line_window2, right_line_window1))

# Draw the lane onto the warped blank image
cv2.fillPoly(window_img, np.int_([left_line_pts]), (0,255, 0))
cv2.fillPoly(window_img, np.int_([right_line_pts]), (0,255, 0))
cv2.fillPoly(window_img, np.int_([mid_line_pts]), (255, 255, 0))
result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)
plt.imshow(result)
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
plt.xlim(0, 1280)
plt.ylim(720, 0)
Out[18]:
(720, 0)
In [19]:
# Define y-value where we want radius of curvature
# I'll choose the maximum y-value, corresponding to the bottom of the image
y_eval = np.max(ploty)
print(y_eval)
left_curverad = ((1 + (2*left_fit[0]*y_eval + left_fit[1])**2)**1.5) / np.absolute(2*left_fit[0])
right_curverad = ((1 + (2*right_fit[0]*y_eval + right_fit[1])**2)**1.5) / np.absolute(2*right_fit[0])
print(left_curverad, right_curverad)
720.0
4059.27675394 3063.5534609
In [20]:
# Define conversions in x and y from pixels space to meters
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
#print(len(ploty))
#print(len(leftx))
#print(len(lefty))
#print(len(left_fitx))
# Fit new polynomials to x,y in world space
left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
# Calculate the new radii of curvature
left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
# Now our radius of curvature is in meters
print(left_curverad, 'm',' ,', right_curverad, 'm')
1319.41141267 m  , 1003.83586415 m
In [21]:
# Create an image to draw the lines on
warp_zero = np.zeros_like(warp_img_h).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

# Recast the x and y points into usable format for cv2.fillPoly()
pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
pts = np.hstack((pts_left, pts_right))

# Draw the lane onto the warped blank image
cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))

Minv = cv2.getPerspectiveTransform(dst_pts, src_pts)

# Warp the blank back to original image space using inverse perspective matrix (Minv)
newwarp = cv2.warpPerspective(color_warp, Minv, (undis_himg.shape[1], undis_himg.shape[0])) 
# Combine the result with the original image
result = cv2.addWeighted(undis_himg, 1, newwarp, 0.3, 0)
plt.imshow(result)
Out[21]:
<matplotlib.image.AxesImage at 0x1cd06cfd898>

Test on Videos

In [22]:
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
In [53]:
def process_image(image):
    # input is RGB image
    # undistort
    undis_img = cv2.undistort(image, mtx, dist, None, mtx)
    # binary image
    s_binary = hls_select(undis_img, thresh=(170, 255))
    sxbinary = abs_sobel_thresh(undis_img, orient='x', sobel_kernel=5, thresh=(20, 100))
    dir_binary = dir_threshold(undis_img, sobel_kernel=15, thresh=(0.7, 1.3))
    combined_binary = np.zeros_like(sxbinary)
    combined_binary[(s_binary == 1) | (sxbinary == 1)] = 1
    
    img_size = (image.shape[1], image.shape[0])
    #src_pts = np.float32([
     #      [204, img_size[1]], #
      #     [585, 460],
       #    [705, 460],
        #   [1110, img_size[1]] ])
    src_pts =  np.float32([
            [((img_size[0] / 6) - 10), img_size[1]],
            [(img_size[0] / 2) - 55, img_size[1] / 2 + 100],
            [(img_size[0] / 2 + 55), img_size[1] / 2 + 100],
            [((img_size[0] * 5 / 6) + 60), img_size[1]] ])
    #dst_pts = np.float32([
      #     [320, img_size[1]],
       #    [320, 0], 
        #   [960, 0], 
         #  [960, img_size[1]] ])
    dst_pts =  np.float32([
            [(img_size[0] / 4), img_size[1]],
            [(img_size[0] / 4), 0],
            [(img_size[0] * 3 / 4), 0],
            [(img_size[0] * 3 / 4), img_size[1]] ])
    
    warp_img = images_warp(combined_binary, src_pts, dst_pts)
    
    histogram = getHistogram(warp_img)

    out_img, left_lane_inds, right_lane_inds = fit_poly(warp_img, histogram)
    
    nonezero = warp_img.nonzero()
    nonezeroy = np.array(nonezero[0])
    nonezerox = np.array(nonezero[1])

    # Extract left and right line pixel positions
    leftx = nonezerox[left_lane_inds]
    lefty = nonezeroy[left_lane_inds]
    rightx = nonezerox[right_lane_inds]
    righty = nonezeroy[right_lane_inds]

    # Fit a second order polynomial to each
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    
    ploty = np.linspace(0, warp_img.shape[0], warp_img.shape[0] )
    y_eval = np.max(ploty)
    
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]

    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/700 # meters per pixel in x dimension
    left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
    right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
    left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
    right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])

    #print(left_curverad, 'm',' ,', right_curverad, 'm')
    
    #out_img[nonezeroy[left_lane_inds], nonezerox[left_lane_inds]] = [255, 0, 0]
    #out_img[nonezeroy[right_lane_inds], nonezerox[right_lane_inds]] = [0, 0, 255]
    
    center_left_fitx = left_fit[0]*y_eval**2 + left_fit[1]*y_eval + left_fit[2]
    center_right_fitx = right_fit[0]*y_eval**2 + right_fit[1]*y_eval + right_fit[2]
    lane_center = np.mean([center_left_fitx, center_right_fitx])
    
    warp_zero = np.zeros_like(warp_img).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    font = cv2.FONT_HERSHEY_SIMPLEX  #使用默认字体

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    pts = np.hstack((pts_left, pts_right))
    #pts1 = np.hstack((pts_left-10, pts_left))
    #pts2 = np.hstack((pts_right, pts_right+10))
    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
    #cv2.fillPoly(color_warp, np.int_([pts1]), (255,0, 0))
    #cv2.fillPoly(color_warp, np.int_([pts2]), (255,0, 0))

    Minv = cv2.getPerspectiveTransform(dst_pts, src_pts)

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (undis_img.shape[1], undis_img.shape[0])) 
    # Combine the result with the original image
    result = cv2.addWeighted(undis_img, 1, newwarp, 0.3, 0)
    
    distance_from_center = np.absolute((img_size[0]/ 2 - lane_center) * xm_per_pix)
    cv2.putText(result, "left curve radius is: {:.0f}m, right curve radius is: {:.0f}m".format(left_curverad,right_curverad ), (50, 50),
                        font, 1.3, (255, 255, 100), 2, cv2.LINE_AA)
    cv2.putText(result, "center shift: {:.2f}m".format(distance_from_center), (50, 90),
                        font, 1.2, (255, 255, 100), 2, cv2.LINE_AA)
    return result

print('Done..')
Done..
In [51]:
testimage_path = './test_images/'
for image in os.listdir(testimage_path):
    image_file = testimage_path + image
    aimg = cv2.imread(image_file) #'test_images/spec.jpg'
    a_img = cv2.cvtColor(aimg, cv2.COLOR_BGR2RGB)
    out = process_image(np.array(a_img))
    f, (ax1) = plt.subplots(1, 1, figsize=(8,8))
    ax1.imshow(out)
In [54]:
white_output = 'project_video_output222.mp4'
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))
[MoviePy] >>>> Building video project_video_output222.mp4
[MoviePy] Writing video project_video_output222.mp4
100%|█████████████████████████████████████████████████████████████████████████████▉| 1260/1261 [05:38<00:00,  3.96it/s]
[MoviePy] Done.
[MoviePy] >>>> Video ready: project_video_output222.mp4 

Wall time: 5min 40s
Out[54]:
In [ ]: